<!DOCTYPE html><html><head><meta charset="utf-8"><style>body {
  width: 45em;
  border: 1px solid #ddd;
  outline: 1300px solid #fff;
  margin: 16px auto;
}

body .markdown-body
{
  padding: 30px;
}

@font-face {
  font-family: octicons-anchor;
  src: url(data:font/woff;charset=utf-8;base64,d09GRgABAAAAAAYcAA0AAAAACjQAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABMAAAABwAAAAca8vGTk9TLzIAAAFMAAAARAAAAFZG1VHVY21hcAAAAZAAAAA+AAABQgAP9AdjdnQgAAAB0AAAAAQAAAAEACICiGdhc3AAAAHUAAAACAAAAAj//wADZ2x5ZgAAAdwAAADRAAABEKyikaNoZWFkAAACsAAAAC0AAAA2AtXoA2hoZWEAAALgAAAAHAAAACQHngNFaG10eAAAAvwAAAAQAAAAEAwAACJsb2NhAAADDAAAAAoAAAAKALIAVG1heHAAAAMYAAAAHwAAACABEAB2bmFtZQAAAzgAAALBAAAFu3I9x/Nwb3N0AAAF/AAAAB0AAAAvaoFvbwAAAAEAAAAAzBdyYwAAAADP2IQvAAAAAM/bz7t4nGNgZGFgnMDAysDB1Ml0hoGBoR9CM75mMGLkYGBgYmBlZsAKAtJcUxgcPsR8iGF2+O/AEMPsznAYKMwIkgMA5REMOXicY2BgYGaAYBkGRgYQsAHyGMF8FgYFIM0ChED+h5j//yEk/3KoSgZGNgYYk4GRCUgwMaACRoZhDwCs7QgGAAAAIgKIAAAAAf//AAJ4nHWMMQrCQBBF/0zWrCCIKUQsTDCL2EXMohYGSSmorScInsRGL2DOYJe0Ntp7BK+gJ1BxF1stZvjz/v8DRghQzEc4kIgKwiAppcA9LtzKLSkdNhKFY3HF4lK69ExKslx7Xa+vPRVS43G98vG1DnkDMIBUgFN0MDXflU8tbaZOUkXUH0+U27RoRpOIyCKjbMCVejwypzJJG4jIwb43rfl6wbwanocrJm9XFYfskuVC5K/TPyczNU7b84CXcbxks1Un6H6tLH9vf2LRnn8Ax7A5WQAAAHicY2BkYGAA4teL1+yI57f5ysDNwgAC529f0kOmWRiYVgEpDgYmEA8AUzEKsQAAAHicY2BkYGB2+O/AEMPCAAJAkpEBFbAAADgKAe0EAAAiAAAAAAQAAAAEAAAAAAAAKgAqACoAiAAAeJxjYGRgYGBhsGFgYgABEMkFhAwM/xn0QAIAD6YBhwB4nI1Ty07cMBS9QwKlQapQW3VXySvEqDCZGbGaHULiIQ1FKgjWMxknMfLEke2A+IJu+wntrt/QbVf9gG75jK577Lg8K1qQPCfnnnt8fX1NRC/pmjrk/zprC+8D7tBy9DHgBXoWfQ44Av8t4Bj4Z8CLtBL9CniJluPXASf0Lm4CXqFX8Q84dOLnMB17N4c7tBo1AS/Qi+hTwBH4rwHHwN8DXqQ30XXAS7QaLwSc0Gn8NuAVWou/gFmnjLrEaEh9GmDdDGgL3B4JsrRPDU2hTOiMSuJUIdKQQayiAth69r6akSSFqIJuA19TrzCIaY8sIoxyrNIrL//pw7A2iMygkX5vDj+G+kuoLdX4GlGK/8Lnlz6/h9MpmoO9rafrz7ILXEHHaAx95s9lsI7AHNMBWEZHULnfAXwG9/ZqdzLI08iuwRloXE8kfhXYAvE23+23DU3t626rbs8/8adv+9DWknsHp3E17oCf+Z48rvEQNZ78paYM38qfk3v/u3l3u3GXN2Dmvmvpf1Srwk3pB/VSsp512bA/GG5i2WJ7wu430yQ5K3nFGiOqgtmSB5pJVSizwaacmUZzZhXLlZTq8qGGFY2YcSkqbth6aW1tRmlaCFs2016m5qn36SbJrqosG4uMV4aP2PHBmB3tjtmgN2izkGQyLWprekbIntJFing32a5rKWCN/SdSoga45EJykyQ7asZvHQ8PTm6cslIpwyeyjbVltNikc2HTR7YKh9LBl9DADC0U/jLcBZDKrMhUBfQBvXRzLtFtjU9eNHKin0x5InTqb8lNpfKv1s1xHzTXRqgKzek/mb7nB8RZTCDhGEX3kK/8Q75AmUM/eLkfA+0Hi908Kx4eNsMgudg5GLdRD7a84npi+YxNr5i5KIbW5izXas7cHXIMAau1OueZhfj+cOcP3P8MNIWLyYOBuxL6DRylJ4cAAAB4nGNgYoAALjDJyIAOWMCiTIxMLDmZedkABtIBygAAAA==) format('woff');
}

.markdown-body {
  -ms-text-size-adjust: 100%;
  -webkit-text-size-adjust: 100%;
  color: #333;
  overflow: hidden;
  font-family: "Helvetica Neue", Helvetica, "Segoe UI", Arial, freesans, sans-serif;
  font-size: 16px;
  line-height: 1.6;
  word-wrap: break-word;
}

.markdown-body a {
  background: transparent;
}

.markdown-body a:active,
.markdown-body a:hover {
  outline: 0;
}

.markdown-body strong {
  font-weight: bold;
}

.markdown-body h1 {
  font-size: 2em;
  margin: 0.67em 0;
}

.markdown-body img {
  border: 0;
}

.markdown-body hr {
  -moz-box-sizing: content-box;
  box-sizing: content-box;
  height: 0;
}

.markdown-body pre {
  overflow: auto;
}

.markdown-body code,
.markdown-body kbd,
.markdown-body pre {
  font-family: monospace, monospace;
  font-size: 1em;
}

.markdown-body input {
  color: inherit;
  font: inherit;
  margin: 0;
}

.markdown-body html input[disabled] {
  cursor: default;
}

.markdown-body input {
  line-height: normal;
}

.markdown-body input[type="checkbox"] {
  -moz-box-sizing: border-box;
  box-sizing: border-box;
  padding: 0;
}

.markdown-body table {
  border-collapse: collapse;
  border-spacing: 0;
}

.markdown-body td,
.markdown-body th {
  padding: 0;
}

.markdown-body * {
  -moz-box-sizing: border-box;
  box-sizing: border-box;
}

.markdown-body input {
  font: 13px/1.4 Helvetica, arial, freesans, clean, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol";
}

.markdown-body a {
  color: #4183c4;
  text-decoration: none;
}

.markdown-body a:hover,
.markdown-body a:focus,
.markdown-body a:active {
  text-decoration: underline;
}

.markdown-body hr {
  height: 0;
  margin: 15px 0;
  overflow: hidden;
  background: transparent;
  border: 0;
  border-bottom: 1px solid #ddd;
}

.markdown-body hr:before {
  display: table;
  content: "";
}

.markdown-body hr:after {
  display: table;
  clear: both;
  content: "";
}

.markdown-body h1,
.markdown-body h2,
.markdown-body h3,
.markdown-body h4,
.markdown-body h5,
.markdown-body h6 {
  margin-top: 15px;
  margin-bottom: 15px;
  line-height: 1.1;
}

.markdown-body h1 {
  font-size: 30px;
}

.markdown-body h2 {
  font-size: 21px;
}

.markdown-body h3 {
  font-size: 16px;
}

.markdown-body h4 {
  font-size: 14px;
}

.markdown-body h5 {
  font-size: 12px;
}

.markdown-body h6 {
  font-size: 11px;
}

.markdown-body blockquote {
  margin: 0;
}

.markdown-body ul,
.markdown-body ol {
  padding: 0;
  margin-top: 0;
  margin-bottom: 0;
}

.markdown-body ol ol,
.markdown-body ul ol {
  list-style-type: lower-roman;
}

.markdown-body ul ul ol,
.markdown-body ul ol ol,
.markdown-body ol ul ol,
.markdown-body ol ol ol {
  list-style-type: lower-alpha;
}

.markdown-body dd {
  margin-left: 0;
}

.markdown-body code {
  font: 12px Consolas, "Liberation Mono", Menlo, Courier, monospace;
}

.markdown-body pre {
  margin-top: 0;
  margin-bottom: 0;
  font: 12px Consolas, "Liberation Mono", Menlo, Courier, monospace;
}

.markdown-body .octicon {
  font: normal normal 16px octicons-anchor;
  line-height: 1;
  display: inline-block;
  text-decoration: none;
  -webkit-font-smoothing: antialiased;
  -moz-osx-font-smoothing: grayscale;
  -webkit-user-select: none;
  -moz-user-select: none;
  -ms-user-select: none;
  user-select: none;
}

.markdown-body .octicon-link:before {
  content: '\f05c';
}

.markdown-body>*:first-child {
  margin-top: 0 !important;
}

.markdown-body>*:last-child {
  margin-bottom: 0 !important;
}

.markdown-body .anchor {
  position: absolute;
  top: 0;
  bottom: 0;
  left: 0;
  display: block;
  padding-right: 6px;
  padding-left: 30px;
  margin-left: -30px;
}

.markdown-body .anchor:focus {
  outline: none;
}

.markdown-body h1,
.markdown-body h2,
.markdown-body h3,
.markdown-body h4,
.markdown-body h5,
.markdown-body h6 {
  position: relative;
  margin-top: 1em;
  margin-bottom: 16px;
  font-weight: bold;
  line-height: 1.4;
}

.markdown-body h1 .octicon-link,
.markdown-body h2 .octicon-link,
.markdown-body h3 .octicon-link,
.markdown-body h4 .octicon-link,
.markdown-body h5 .octicon-link,
.markdown-body h6 .octicon-link {
  display: none;
  color: #000;
  vertical-align: middle;
}

.markdown-body h1:hover .anchor,
.markdown-body h2:hover .anchor,
.markdown-body h3:hover .anchor,
.markdown-body h4:hover .anchor,
.markdown-body h5:hover .anchor,
.markdown-body h6:hover .anchor {
  padding-left: 8px;
  margin-left: -30px;
  line-height: 1;
  text-decoration: none;
}

.markdown-body h1:hover .anchor .octicon-link,
.markdown-body h2:hover .anchor .octicon-link,
.markdown-body h3:hover .anchor .octicon-link,
.markdown-body h4:hover .anchor .octicon-link,
.markdown-body h5:hover .anchor .octicon-link,
.markdown-body h6:hover .anchor .octicon-link {
  display: inline-block;
}

.markdown-body h1 {
  padding-bottom: 0.3em;
  font-size: 2.25em;
  line-height: 1.2;
  border-bottom: 1px solid #eee;
}

.markdown-body h2 {
  padding-bottom: 0.3em;
  font-size: 1.75em;
  line-height: 1.225;
  border-bottom: 1px solid #eee;
}

.markdown-body h3 {
  font-size: 1.5em;
  line-height: 1.43;
}

.markdown-body h4 {
  font-size: 1.25em;
}

.markdown-body h5 {
  font-size: 1em;
}

.markdown-body h6 {
  font-size: 1em;
  color: #777;
}

.markdown-body p,
.markdown-body blockquote,
.markdown-body ul,
.markdown-body ol,
.markdown-body dl,
.markdown-body table,
.markdown-body pre {
  margin-top: 0;
  margin-bottom: 16px;
}

.markdown-body hr {
  height: 4px;
  padding: 0;
  margin: 16px 0;
  background-color: #e7e7e7;
  border: 0 none;
}

.markdown-body ul,
.markdown-body ol {
  padding-left: 2em;
}

.markdown-body ul ul,
.markdown-body ul ol,
.markdown-body ol ol,
.markdown-body ol ul {
  margin-top: 0;
  margin-bottom: 0;
}

.markdown-body li>p {
  margin-top: 16px;
}

.markdown-body dl {
  padding: 0;
}

.markdown-body dl dt {
  padding: 0;
  margin-top: 16px;
  font-size: 1em;
  font-style: italic;
  font-weight: bold;
}

.markdown-body dl dd {
  padding: 0 16px;
  margin-bottom: 16px;
}

.markdown-body blockquote {
  padding: 0 15px;
  color: #777;
  border-left: 4px solid #ddd;
}

.markdown-body blockquote>:first-child {
  margin-top: 0;
}

.markdown-body blockquote>:last-child {
  margin-bottom: 0;
}

.markdown-body table {
  display: block;
  width: 100%;
  overflow: auto;
  word-break: normal;
  word-break: keep-all;
}

.markdown-body table th {
  font-weight: bold;
}

.markdown-body table th,
.markdown-body table td {
  padding: 6px 13px;
  border: 1px solid #ddd;
}

.markdown-body table tr {
  background-color: #fff;
  border-top: 1px solid #ccc;
}

.markdown-body table tr:nth-child(2n) {
  background-color: #f8f8f8;
}

.markdown-body img {
  max-width: 100%;
  -moz-box-sizing: border-box;
  box-sizing: border-box;
}

.markdown-body code {
  padding: 0;
  padding-top: 0.2em;
  padding-bottom: 0.2em;
  margin: 0;
  font-size: 85%;
  background-color: rgba(0,0,0,0.04);
  border-radius: 3px;
}

.markdown-body code:before,
.markdown-body code:after {
  letter-spacing: -0.2em;
  content: "\00a0";
}

.markdown-body pre>code {
  padding: 0;
  margin: 0;
  font-size: 100%;
  word-break: normal;
  white-space: pre;
  background: transparent;
  border: 0;
}

.markdown-body .highlight {
  margin-bottom: 16px;
}

.markdown-body .highlight pre,
.markdown-body pre {
  padding: 16px;
  overflow: auto;
  font-size: 85%;
  line-height: 1.45;
  background-color: #f7f7f7;
  border-radius: 3px;
}

.markdown-body .highlight pre {
  margin-bottom: 0;
  word-break: normal;
}

.markdown-body pre {
  word-wrap: normal;
}

.markdown-body pre code {
  display: inline;
  max-width: initial;
  padding: 0;
  margin: 0;
  overflow: initial;
  line-height: inherit;
  word-wrap: normal;
  background-color: transparent;
  border: 0;
}

.markdown-body pre code:before,
.markdown-body pre code:after {
  content: normal;
}

.markdown-body .highlight {
  background: #fff;
}

.markdown-body .highlight .h {
  color: #333;
  font-style: normal;
  font-weight: normal;
}

.markdown-body .highlight .mf,
.markdown-body .highlight .mh,
.markdown-body .highlight .mi,
.markdown-body .highlight .mo,
.markdown-body .highlight .il,
.markdown-body .highlight .m {
  color: #945277;
}

.markdown-body .highlight .s,
.markdown-body .highlight .sb,
.markdown-body .highlight .sc,
.markdown-body .highlight .sd,
.markdown-body .highlight .s2,
.markdown-body .highlight .se,
.markdown-body .highlight .sh,
.markdown-body .highlight .si,
.markdown-body .highlight .sx,
.markdown-body .highlight .s1 {
  color: #df5000;
}

.markdown-body .highlight .kc,
.markdown-body .highlight .kd,
.markdown-body .highlight .kn,
.markdown-body .highlight .kp,
.markdown-body .highlight .kr,
.markdown-body .highlight .kt,
.markdown-body .highlight .k,
.markdown-body .highlight .o {
  font-weight: bold;
}

.markdown-body .highlight .kt {
  color: #458;
}

.markdown-body .highlight .c,
.markdown-body .highlight .cm,
.markdown-body .highlight .c1 {
  color: #998;
  font-style: italic;
}

.markdown-body .highlight .cp,
.markdown-body .highlight .cs,
.markdown-body .highlight .cp .h {
  color: #999;
  font-weight: bold;
}

.markdown-body .highlight .cs {
  font-style: italic;
}

.markdown-body .highlight .n {
  color: #333;
}

.markdown-body .highlight .na,
.markdown-body .highlight .nv,
.markdown-body .highlight .vc,
.markdown-body .highlight .vg,
.markdown-body .highlight .vi {
  color: #008080;
}

.markdown-body .highlight .nb {
  color: #0086B3;
}

.markdown-body .highlight .nc {
  color: #458;
  font-weight: bold;
}

.markdown-body .highlight .no {
  color: #094e99;
}

.markdown-body .highlight .ni {
  color: #800080;
}

.markdown-body .highlight .ne {
  color: #990000;
  font-weight: bold;
}

.markdown-body .highlight .nf {
  color: #945277;
  font-weight: bold;
}

.markdown-body .highlight .nn {
  color: #555;
}

.markdown-body .highlight .nt {
  color: #000080;
}

.markdown-body .highlight .err {
  color: #a61717;
  background-color: #e3d2d2;
}

.markdown-body .highlight .gd {
  color: #000;
  background-color: #fdd;
}

.markdown-body .highlight .gd .x {
  color: #000;
  background-color: #faa;
}

.markdown-body .highlight .ge {
  font-style: italic;
}

.markdown-body .highlight .gr {
  color: #aa0000;
}

.markdown-body .highlight .gh {
  color: #999;
}

.markdown-body .highlight .gi {
  color: #000;
  background-color: #dfd;
}

.markdown-body .highlight .gi .x {
  color: #000;
  background-color: #afa;
}

.markdown-body .highlight .go {
  color: #888;
}

.markdown-body .highlight .gp {
  color: #555;
}

.markdown-body .highlight .gs {
  font-weight: bold;
}

.markdown-body .highlight .gu {
  color: #800080;
  font-weight: bold;
}

.markdown-body .highlight .gt {
  color: #aa0000;
}

.markdown-body .highlight .ow {
  font-weight: bold;
}

.markdown-body .highlight .w {
  color: #bbb;
}

.markdown-body .highlight .sr {
  color: #017936;
}

.markdown-body .highlight .ss {
  color: #8b467f;
}

.markdown-body .highlight .bp {
  color: #999;
}

.markdown-body .highlight .gc {
  color: #999;
  background-color: #EAF2F5;
}

.markdown-body kbd {
  background-color: #e7e7e7;
  background-image: -webkit-linear-gradient(#fefefe, #e7e7e7);
  background-image: linear-gradient(#fefefe, #e7e7e7);
  background-repeat: repeat-x;
  display: inline-block;
  padding: 3px 5px;
  font: 11px Consolas, "Liberation Mono", Menlo, Courier, monospace;
  line-height: 10px;
  color: #000;
  border: 1px solid #cfcfcf;
  border-radius: 2px;
}

.markdown-body .highlight .pl-coc,
.markdown-body .highlight .pl-entm,
.markdown-body .highlight .pl-eoa,
.markdown-body .highlight .pl-mai .pl-sf,
.markdown-body .highlight .pl-pdv,
.markdown-body .highlight .pl-sc,
.markdown-body .highlight .pl-sr,
.markdown-body .highlight .pl-v,
.markdown-body .highlight .pl-vpf {
  color: #0086b3;
}

.markdown-body .highlight .pl-eoac,
.markdown-body .highlight .pl-mdht,
.markdown-body .highlight .pl-mi1,
.markdown-body .highlight .pl-mri,
.markdown-body .highlight .pl-va,
.markdown-body .highlight .pl-vpu {
  color: #008080;
}

.markdown-body .highlight .pl-c,
.markdown-body .highlight .pl-pdc {
  color: #b4b7b4;
  font-style: italic;
}

.markdown-body .highlight .pl-k,
.markdown-body .highlight .pl-ko,
.markdown-body .highlight .pl-kolp,
.markdown-body .highlight .pl-mc,
.markdown-body .highlight .pl-mr,
.markdown-body .highlight .pl-ms,
.markdown-body .highlight .pl-s,
.markdown-body .highlight .pl-sok,
.markdown-body .highlight .pl-st {
  color: #6e5494;
}

.markdown-body .highlight .pl-ef,
.markdown-body .highlight .pl-enf,
.markdown-body .highlight .pl-enm,
.markdown-body .highlight .pl-entc,
.markdown-body .highlight .pl-eoi,
.markdown-body .highlight .pl-sf,
.markdown-body .highlight .pl-smc {
  color: #d12089;
}

.markdown-body .highlight .pl-ens,
.markdown-body .highlight .pl-eoai,
.markdown-body .highlight .pl-kos,
.markdown-body .highlight .pl-mh .pl-pdh,
.markdown-body .highlight .pl-mp,
.markdown-body .highlight .pl-pde,
.markdown-body .highlight .pl-stp {
  color: #458;
}

.markdown-body .highlight .pl-enti {
  color: #d12089;
  font-weight: bold;
}

.markdown-body .highlight .pl-cce,
.markdown-body .highlight .pl-enc,
.markdown-body .highlight .pl-kou,
.markdown-body .highlight .pl-mq {
  color: #f93;
}

.markdown-body .highlight .pl-mp1 .pl-sf {
  color: #458;
  font-weight: bold;
}

.markdown-body .highlight .pl-cos,
.markdown-body .highlight .pl-ent,
.markdown-body .highlight .pl-md,
.markdown-body .highlight .pl-mdhf,
.markdown-body .highlight .pl-ml,
.markdown-body .highlight .pl-pdc1,
.markdown-body .highlight .pl-pds,
.markdown-body .highlight .pl-s1,
.markdown-body .highlight .pl-scp,
.markdown-body .highlight .pl-sol {
  color: #df5000;
}

.markdown-body .highlight .pl-c1,
.markdown-body .highlight .pl-cn,
.markdown-body .highlight .pl-pse,
.markdown-body .highlight .pl-pse .pl-s2,
.markdown-body .highlight .pl-vi {
  color: #a31515;
}

.markdown-body .highlight .pl-mb,
.markdown-body .highlight .pl-pdb {
  color: #df5000;
  font-weight: bold;
}

.markdown-body .highlight .pl-mi,
.markdown-body .highlight .pl-pdi {
  color: #6e5494;
  font-style: italic;
}

.markdown-body .highlight .pl-ms1 {
  background-color: #f5f5f5;
}

.markdown-body .highlight .pl-mdh,
.markdown-body .highlight .pl-mdi {
  font-weight: bold;
}

.markdown-body .highlight .pl-mdr {
  color: #0086b3;
  font-weight: bold;
}

.markdown-body .highlight .pl-s2 {
  color: #333;
}

.markdown-body .highlight .pl-ii {
  background-color: #df5000;
  color: #fff;
}

.markdown-body .highlight .pl-ib {
  background-color: #f93;
}

.markdown-body .highlight .pl-id {
  background-color: #a31515;
  color: #fff;
}

.markdown-body .highlight .pl-iu {
  background-color: #b4b7b4;
}

.markdown-body .highlight .pl-mo {
  color: #969896;
}

.markdown-body .task-list-item {
  list-style-type: none;
}

.markdown-body .task-list-item+.task-list-item {
  margin-top: 3px;
}

.markdown-body .task-list-item input {
  float: left;
  margin: 0.3em 0 0.25em -1.6em;
  vertical-align: middle;
}</style><title>Nutch教程——URLNormalizer源码详解 by 逼格DATA</title></head><body><article class="markdown-body"><h2>
<a id="user-content-nutch教程urlnormalizer源码详解-by-逼格data" class="anchor" href="#nutch%E6%95%99%E7%A8%8Burlnormalizer%E6%BA%90%E7%A0%81%E8%AF%A6%E8%A7%A3-by-%E9%80%BC%E6%A0%BCdata" aria-hidden="true"><span class="octicon octicon-link"></span></a>Nutch教程——URLNormalizer源码详解 by 逼格DATA</h2>

<p>本教程由<a href="http://nutcher.org/book/">逼格DATA</a>提供,未经允许,禁止转载.同时托管在
<a href="https://github.com/CrawlScript/nutcher">github</a>上.</p>

<p>可加入nutcher的bbs进行讨论：<a href="http://nutcher.org">Nutch开发者</a></p>

<p>URL正规化（URLNormalize）对大多数网络爬虫来说是一个非常重要的流程,大部分爬虫的去重机制都是基于URL的去重,在实际中,不同的URL有可能对应同一个网页,例如下面的几个URL：</p>

<ul>
<li><a href="http://nutcher.org/">http://nutcher.org/</a></li>
<li><a href="http://nutcher.org/abc/../">http://nutcher.org/abc/../</a></li>
<li><a href="http://nutcher.org/./">http://nutcher.org/./</a></li>
</ul>

<p>这些URL其实都指向<a href="http://nutcher.org/">http://nutcher.org/</a>,在文件路径中,../表示上级目录,
./表示当前目录,所以/abc/../和/./都等价于/.如果爬虫没有URL正规化机制,
爬虫会认为<a href="http://nutcher.org/">http://nutcher.org/</a>、<a href="http://nutcher.org/abc/../">http://nutcher.org/abc/../</a>、<a href="http://nutcher.org/./">http://nutcher.org/./</a>是三个不同的页面,而三个URL实际都指向<a href="http://nutcher.org/">http://nutcher.org/</a>,因此会将<a href="http://nutcher.org/">http://nutcher.org/</a>爬取三次.</p>

<p>可以看出,URL正规化的主要目的就是防止网页的重复采集.这里我们介绍Nutch的URL正规化组件之一,
urlnormalizer-basic,该组件在nutch中对应的类是<a href="https://github.com/CrawlScript/nutcher/blob/master/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/BasicURLNormalizer.java">BasicURLNormalizer.java</a>，这里我们用源码注释的方式描述urlnormalizer-basic的工作机制：</p>

<pre><code>package org.apache.nutch.net.urlnormalizer.basic;

import java.net.URL;
import java.net.MalformedURLException;

// Slf4j Logging imports
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

// Nutch imports
import org.apache.nutch.net.URLNormalizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.oro.text.regex.*;

/**
 * Converts URLs to a normal form:
 * &lt;ul&gt;
 * &lt;li&gt;remove dot segments in path: &lt;code&gt;/./&lt;/code&gt; or &lt;code&gt;/../&lt;/code&gt;&lt;/li&gt;
 * &lt;li&gt;remove default ports, e.g. 80 for protocol &lt;code&gt;http://&lt;/code&gt;&lt;/li&gt;
 * &lt;/ul&gt;
 * 将URL转换为正规的形式
 * 移除URL中类似/./和/../的部分
 * 移除默认端口，例如对于http协议，移除URL中指定的80端口
 */
public class BasicURLNormalizer extends Configured implements URLNormalizer {
    public static final Logger LOG = LoggerFactory.getLogger(BasicURLNormalizer.class);

    /*这里使用Perl规范的正则，借助了org.apache.oro.text.regex.Perl5Compiler类*/
    private Perl5Compiler compiler = new Perl5Compiler();
    private ThreadLocal&lt;Perl5Matcher&gt; matchers = new ThreadLocal&lt;Perl5Matcher&gt;() {
        protected Perl5Matcher initialValue() {
          return new Perl5Matcher();
        }
      };
    private final Rule relativePathRule;
    private final Rule leadingRelativePathRule;
    private final Rule currentPathRule;
    private final Rule adjacentSlashRule;

    private final static java.util.regex.Pattern hasNormalizablePattern = java.util.regex.Pattern.compile("/\\.?\\.?/");

    private Configuration conf;

    public BasicURLNormalizer() {
      try {
        // this pattern tries to find spots like "/xx/../" in the url, which
        // could be replaced by "/" xx consists of chars, different then "/"
        // (slash) and needs to have at least one char different from "."
        // 这里希望从URL中找到匹配/xx/../的部分,替换为/
        // xx中的每个字符不能是'/'，且至少出现一个不为'.'的字符
        relativePathRule = new Rule();
        relativePathRule.pattern = (Perl5Pattern)
          compiler.compile("(/[^/]*[^/.]{1}[^/]*/\\.\\./)",
                           Perl5Compiler.READ_ONLY_MASK);
        relativePathRule.substitution = new Perl5Substitution("/");


        // this pattern tries to find spots like leading "/../" in the url,
        // which could be replaced by "/"
        // 如果URL中的路径，以/../起始,将其替换为/
        leadingRelativePathRule = new Rule();
        leadingRelativePathRule.pattern = (Perl5Pattern)
          compiler.compile("^(/\\.\\./)+", Perl5Compiler.READ_ONLY_MASK);
        leadingRelativePathRule.substitution = new Perl5Substitution("/");


        // this pattern tries to find spots like "/./" in the url,
        // which could be replaced by "/"
        // 这里希望从URL中找到匹配/./的部分,替换为/
        currentPathRule = new Rule();
        currentPathRule.pattern = (Perl5Pattern)
          compiler.compile("(/\\./)", Perl5Compiler.READ_ONLY_MASK);
        currentPathRule.substitution = new Perl5Substitution("/");


        // this pattern tries to find spots like "xx//yy" in the url,
        // which could be replaced by a "/"
        // 这里希望从URL中找到匹配//的部分,替换为/
        adjacentSlashRule = new Rule();
        adjacentSlashRule.pattern = (Perl5Pattern)
          compiler.compile("/{2,}", Perl5Compiler.READ_ONLY_MASK);
        adjacentSlashRule.substitution = new Perl5Substitution("/");

      } catch (MalformedPatternException e) {
        throw new RuntimeException(e);
      }
    }

    public String normalize(String urlString, String scope)
            throws MalformedURLException {
        if ("".equals(urlString))                     // permit empty
            return urlString;

        urlString = urlString.trim();                 // remove extra spaces

        URL url = new URL(urlString);

        String protocol = url.getProtocol();
        String host = url.getHost();
        int port = url.getPort();
        String file = url.getFile();

        boolean changed = false;

        if (!urlString.startsWith(protocol))        // protocol was lowercased
            changed = true;

        if ("http".equals(protocol) || "https".equals(protocol) || "ftp".equals(protocol)) {

            if (host != null) {
                String newHost = host.toLowerCase();    // lowercase host
                if (!host.equals(newHost)) {
                    host = newHost;
                    changed = true;
                }
            }

            if (port == url.getDefaultPort()) {       // uses default port 如果URL中出现端口信息，检查是否为URL对应协议默认端口
                port = -1;                              // so don't specify it
                changed = true;
            }

            if (file == null || "".equals(file)) {    // add a slash
                file = "/";
                changed = true;
            }

            if (url.getRef() != null) {                 // remove the ref
                changed = true;
            }

            // check for unnecessary use of "/../"
            String file2 = substituteUnnecessaryRelativePaths(file);

            if (!file.equals(file2)) {
                changed = true;
                file = file2;
            }

        }

        if (changed)
            urlString = new URL(protocol, host, port, file).toString();

        return urlString;
    }

    private String substituteUnnecessaryRelativePaths(String file) {

        if (!hasNormalizablePattern.matcher(file).find())
            return file;

        String fileWorkCopy = file;
        int oldLen = file.length();
        int newLen = oldLen - 1;

        // All substitutions will be done step by step, to ensure that certain
        // constellations will be normalized, too
        //
        // URL正规化会一步一步进行，下面给出一个例子
        //
        // For example: "/aa/bb/../../cc/../foo.html will be normalized in the
        // following manner:
        //   "/aa/bb/../../cc/../foo.html"
        //   "/aa/../cc/../foo.html"
        //   "/cc/../foo.html"
        //   "/foo.html"
        //
        // The normalization also takes care of leading "/../", which will be
        // replaced by "/", because this is a rather a sign of bad webserver
        // configuration than of a wanted link.  For example, urls like
        // "http://www.foo.com/../" should return a http 404 error instead of
        // redirecting to "http://www.foo.com".
        //
        Perl5Matcher matcher = (Perl5Matcher)matchers.get();

        while (oldLen != newLen) {
            // substitue first occurence of "/xx/../" by "/"
            oldLen = fileWorkCopy.length();
            fileWorkCopy = Util.substitute
              (matcher, relativePathRule.pattern,
               relativePathRule.substitution, fileWorkCopy, 1);

            // remove leading "/../"
            fileWorkCopy = Util.substitute
              (matcher, leadingRelativePathRule.pattern,
               leadingRelativePathRule.substitution, fileWorkCopy, 1);

            // remove unnecessary "/./"
            fileWorkCopy = Util.substitute
            (matcher, currentPathRule.pattern,
                    currentPathRule.substitution, fileWorkCopy, 1);


            // collapse adjacent slashes with "/"
            fileWorkCopy = Util.substitute
            (matcher, adjacentSlashRule.pattern,
              adjacentSlashRule.substitution, fileWorkCopy, 1);

            newLen = fileWorkCopy.length();
        }

        return fileWorkCopy;
    }


    /**
     * Class which holds a compiled pattern and its corresponding substition
     * string.
     */
    private static class Rule {
        public Perl5Pattern pattern;
        public Perl5Substitution substitution;
    }


  public void setConf(Configuration conf) {
    this.conf = conf;
  }

  public Configuration getConf() {
    return this.conf;
  }

}
</code></pre>

<p>本教程由<a href="http://nutcher.org/book/">逼格DATA</a>提供，未经允许，禁止转载。同时托管在
<a href="https://github.com/CrawlScript/nutcher">github</a>上。</p>

<p>可加入nutcher的bbs进行讨论：<a href="http://nutcher.org">Nutch开发者</a></p>
</article>

<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "//hm.baidu.com/hm.js?5e05d3351c34aa9e8bbecba8f2eaafb6";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script>

</body></html>
